{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "6266546c-eac0-4018-83c8-30db5254ad7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "导包\n",
    "\"\"\"\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "39ef3a99-a838-4658-a6db-a499a3d5424d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Requests HTTP Library\n",
      "~~~~~~~~~~~~~~~~~~~~~\n",
      "\n",
      "Requests is an HTTP library, written in Python, for human beings.\n",
      "Basic GET usage:\n",
      "\n",
      "   >>> import requests\n",
      "   >>> r = requests.get('https://www.python.org')\n",
      "   >>> r.status_code\n",
      "   200\n",
      "   >>> b'Python is a programming language' in r.content\n",
      "   True\n",
      "\n",
      "... or POST:\n",
      "\n",
      "   >>> payload = dict(key1='value1', key2='value2')\n",
      "   >>> r = requests.post('https://httpbin.org/post', data=payload)\n",
      "   >>> print(r.text)\n",
      "   {\n",
      "     ...\n",
      "     \"form\": {\n",
      "       \"key1\": \"value1\",\n",
      "       \"key2\": \"value2\"\n",
      "     },\n",
      "     ...\n",
      "   }\n",
      "\n",
      "The other HTTP methods are supported - see `requests.api`. Full documentation\n",
      "is at <https://requests.readthedocs.io>.\n",
      "\n",
      ":copyright: (c) 2017 by Kenneth Reitz.\n",
      ":license: Apache 2.0, see LICENSE for more details.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 查看reqeust文档\n",
    "print(requests.__doc__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "dcde8376-c472-4309-9584-92ac0e5acba9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<module 'requests' from '/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/requests/__init__.py'>\n",
      "\n",
      "////////////////////////////////////////////////////////////////////////////////////////////////\n",
      "\n",
      "['ConnectTimeout', 'ConnectionError', 'DependencyWarning', 'FileModeWarning', 'HTTPError', 'JSONDecodeError', 'NullHandler', 'PreparedRequest', 'ReadTimeout', 'Request', 'RequestException', 'RequestsDependencyWarning', 'Response', 'Session', 'Timeout', 'TooManyRedirects', 'URLRequired', '__author__', '__author_email__', '__build__', '__builtins__', '__cached__', '__cake__', '__copyright__', '__description__', '__doc__', '__file__', '__license__', '__loader__', '__name__', '__package__', '__path__', '__spec__', '__title__', '__url__', '__version__', '_check_cryptography', '_internal_utils', 'adapters', 'api', 'auth', 'certs', 'chardet_version', 'charset_normalizer_version', 'check_compatibility', 'codes', 'compat', 'cookies', 'delete', 'exceptions', 'get', 'head', 'hooks', 'logging', 'models', 'options', 'packages', 'patch', 'post', 'put', 'request', 'session', 'sessions', 'ssl', 'status_codes', 'structures', 'urllib3', 'utils', 'warnings']\n",
      "\n",
      "////////////////////////////////////////////////////////////////////////////////////////////////\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 了解新包、模块的三板斧\n",
    "print(requests)\n",
    "\n",
    "print('\\n////////////////////////////////////////////////////////////////////////////////////////////////\\n')\n",
    "\n",
    "print(dir(requests))\n",
    "\n",
    "print('\\n////////////////////////////////////////////////////////////////////////////////////////////////\\n')\n",
    "\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "8f7a8870-3a58-4369-8100-df8a61b99e5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.31.0'"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看reqeusts版本\n",
    "requests.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "493c3724-188f-47f8-9cf3-e4e254a50fbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = 'http://www.baidu.com'\n",
    "his = ['/s?rtt=1&bsst=1&cl=2&tn=news&ie=utf-8&word=ESG']\n",
    "url = base_url + his[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "5e54c2f0-6dcf-4f75-83f5-9753e6af23b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 发起请求\n",
    "page = requests.get(url=url)\n",
    "# 查看状态码\n",
    "page.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "5e8f0d86-3d75-46c7-8acf-83254fad7e5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 自动匹配encoding\n",
    "page.encoding = page.apparent_encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "096593b6-ac81-42eb-aea2-2f12194972c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(markup=page.text, features='html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "ca551b65-b10e-46ec-8904-95fe0063bad9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "百度资讯搜索_ESG\n"
     ]
    }
   ],
   "source": [
    "print(soup.find(name='title').get_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "7597826e-17ac-4264-b6b2-80dbca83331c",
   "metadata": {},
   "outputs": [],
   "source": [
    "cell = soup.find(name='div', attrs={'id': 'content_left'}).find_all(name='div', attrs={'class': 'result-op c-container xpath-log new-pmd'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "eb65f03c-cb7f-4049-b8bc-90921e3509d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "cell_list = soup.find_all(name='a', attrs={'class': 'news-title-font_1xS-F'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "64b24e22-a504-48e7-aa54-a7a5164aa232",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(file='passage01.txt', mode='w', encoding='utf-8') as f:\n",
    "    for cell_title in cell_list:\n",
    "        print(cell_title['aria-label'][3:], '\\tURL:', cell_title['href'], file=f)\n",
    "        print(file=f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
